package net.bwie.realtime.dws.log.job;

import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONObject;
import org.apache.flink.api.common.functions.FilterFunction;
import org.apache.flink.streaming.api.datastream.DataStream;
import org.apache.flink.streaming.api.datastream.SideOutputDataStream;
import org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.streaming.api.functions.ProcessFunction;
import org.apache.flink.util.Collector;
import org.apache.flink.util.OutputTag;
import utils.KafkaUtil;

public class VehicleAppEtlLog {

    public static void main(String[] args) throws Exception{

        //1.执行环境
        StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();

        //2.读取数据
        DataStream<String> trafficStream = KafkaUtil.consumerKafka(env, "traffic_monitor_data");
//        trafficStream.print("kafka");

        //3.数据处理
        DataStream<String> etlStream = processLog(trafficStream);
        etlStream.print("kafka") ;

        //4.数据输出
        KafkaUtil.producerKafka(etlStream, "dwd-traffic-monitor-log");

        //5. 执行
        env.execute("VehicleAppEtlLog");

    }

    /**
     * 数据处理:将车速小于等于10和大于200的数据过滤掉
     * @param stream
     * @return
     */
    private static DataStream<String> processLog(DataStream<String> stream) {

        //1.数据清洗
        DataStream<String> cleanedStream = AppLogCleaned(stream);

        //2.车速过滤
        DataStream<String> resultStream = CarFilterStream(cleanedStream);

        return resultStream;
    }

    private static DataStream<String> CarFilterStream(DataStream<String> stream) {
        SingleOutputStreamOperator<String> speedStream = stream.filter(new FilterFunction<String>() {
            @Override
            public boolean filter(String value) throws Exception {
                // 解析json
                JSONObject jsonObject = JSON.parseObject(value);
                Integer speed = jsonObject.getInteger("speed");
                if (speed > 10 && speed < 200) {
                    return true;
                }
                return false;
            }
        });
        return speedStream;
    }

    private static DataStream<String> AppLogCleaned(DataStream<String> stream) {

        //脏数据侧边流输出标记
        final OutputTag<String> dirtyTag = new OutputTag<String>("dirty-log"){};

        // 数据清洗处理
        SingleOutputStreamOperator<String> processStream = stream.process(new ProcessFunction<String, String>() {
            @Override
            public void processElement(String value, Context ctx, Collector<String> out) throws Exception {
                try {
                    //解析json数据
                    JSON.parseObject(value);
                    //没有异常，正常输出
                    out.collect(value);
                } catch (Exception e) {
                    //捕获异常，侧边流输出数据
                    ctx.output(dirtyTag, value);
                }
            }
        });

        // 脏数据侧边流输出
        SideOutputDataStream<String> dirtyStream = processStream.getSideOutput(dirtyTag);
        KafkaUtil.producerKafka(dirtyStream,"dwd-traffic-monitor-dirty-log");

        return processStream;
    }
}
